package com.cnap.multiclouldmgmt.service.impl;

import cn.hutool.extra.spring.SpringUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.cnap.multiclouldmgmt.model.CommonResult;
import com.cnap.multiclouldmgmt.model.enums.*;
import com.cnap.multiclouldmgmt.model.flavor.*;
import com.cnap.multiclouldmgmt.model.gpu.FlavorGPUInfoWithImageDisk;
import com.cnap.multiclouldmgmt.model.gpu.GpuFlavorInfo;
import com.cnap.multiclouldmgmt.model.gpu.GpuSpecInfo;
import com.cnap.multiclouldmgmt.model.k8sModels.K8sClusterBasicModel;
import com.cnap.multiclouldmgmt.model.k8sModels.K8sClusterModel;
import com.cnap.multiclouldmgmt.model.microSvc.MicroServicePO;
import com.cnap.multiclouldmgmt.model.privatecloud.PrivateVmCreateModel;
import com.cnap.multiclouldmgmt.model.quota.UpdateUsedQuotaReq;
import com.cnap.multiclouldmgmt.model.quota.UserQuota;
import com.cnap.multiclouldmgmt.model.quota.UserQuotaResult;
import com.cnap.multiclouldmgmt.model.vm.*;
import com.cnap.multiclouldmgmt.model.vmScale.NotifyClusterInfoDTO;
import com.cnap.multiclouldmgmt.repository.cnap.MicroSvcMapper;
import com.cnap.multiclouldmgmt.repository.master.FlavorInfoMapper;
import com.cnap.multiclouldmgmt.repository.master.VmCreateInfoMapper;
import com.cnap.multiclouldmgmt.service.*;
import com.cnap.multiclouldmgmt.util.CnapCloudMgmtConstants;
import com.cnap.multiclouldmgmt.util.DateUtils;
import com.cnap.multiclouldmgmt.util.TokenUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.http.HttpHeaders;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;

import java.util.*;

@Service("VmResourceServiceImpl")
public class VmResourceServiceImpl implements VmResourceService {

    private final static Logger LOGGER = LoggerFactory.getLogger(VmResourceServiceImpl.class);

    private static final String UPDATE_STAGE_PATH = "/sys/stage/update";

    @Autowired
    private VmCreateInfoMapper vmCreateInfoMapper;

    @Autowired
    private MicroSvcMapper microSvcMapper;

    @Autowired
    private MultiCloudMgmtService multiCloudMgmtService;

    @Autowired
    private K8sClusterService k8sClusterService;

    @Autowired
    private CnapSshService cnapSshService;

    @Autowired
    private CacheService cacheService;

    @Autowired
    private FlavorInfoMapper flavorInfoMapper;

    private static final String ALG_URL = "http://cnap-algorithm-svc.default.svc.cluster.local:8808";

    private static final String SCALE_URL = "http://cnap-mgmt-be-svc.default.svc.cluster.local:8080";

    private static final String VM_CONTROLER_URL = "http://cnap-cloud-mgmt-be-svc.default.svc.cluster.local:8080";

    private static final String CNP_AUTH_URL = "http://cnp-auth-svc.default.svc.cluster.local:8080";

    //true 可以添加集群，false不可以添加
    @Override
    public boolean checkUserK8sClusterQuota(String usrName) {

        //admin肯定可以新增
        if (StringUtils.equalsIgnoreCase(usrName,"admin")){
            return true;
        }

        if (StringUtils.isBlank(usrName)){
            return false;
        }

        try {
            String url = CNP_AUTH_URL + "/quota/getUserQuota";

            Map<String, String> param = new HashMap<>();
            param.put("userName", usrName);
            String quotaStr = doHttpPostReq(url, new StringEntity(JSONObject.toJSONString(param), "UTF-8"));

            LOGGER.warn("checkUserK8sClusterQuota send req userName={} result={}", usrName, quotaStr);

            if (StringUtils.isNotBlank(quotaStr)) {
                UserQuotaResult result = JSONObject.parseObject(quotaStr, UserQuotaResult.class);

                int remainK8sQuota = Integer.parseInt(result.getData().getK8sClusterSize());

                if (remainK8sQuota > 0){
                    return true;
                }
            }
        }catch (Exception e){
            LOGGER.error("checkUserK8sClusterQuota failed",e);
        }



        return false;
    }

    @Override
    public void updateK8sClusterUseQuota(String k8sclusterName,String usrName, String handler, int k8sChangeNum) {
        String path = "/quota/updateUsedQuota";

        String url = CNP_AUTH_URL + path;

        UpdateUsedQuotaReq updateUsedQuotaReq
                = new UpdateUsedQuotaReq(k8sChangeNum,usrName,handler);
        String param = JSONObject.toJSONString(updateUsedQuotaReq);
        LOGGER.warn("updateK8sClusterUseQuota send req k8sClusterName={} param={}",k8sclusterName,param);
        doHttpPostReq(url,new StringEntity(JSONObject.toJSONString(updateUsedQuotaReq),"UTF-8"));

    }

    private boolean isNumeric(String str) {
        for (int i = 0; i < str.length(); i++) {
            System.out.println(str.charAt(i));
            if (!Character.isDigit(str.charAt(i))) {
                return false;
            }
        }
        return true;
    }

    @Override
    public void addVmK8sCluster(PrivateVmCreateModel privateVmCreateModel) {

        LOGGER.warn("addVmK8sCluster begin param={}",JSONObject.toJSONString(privateVmCreateModel));

        VmCreateModel vmCreateModel = new VmCreateModel();

        //默认虚拟机已经把端口和机器都弄好了
        vmCreateModel.setType(StageTypeEnum.SAFE_PORT.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.TOW_FOUR.getStepOrder());
        vmCreateModel.setOrder(StageDetailTypeEnum.TOW_FOUR.getOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.TOW_FOUR.getDetailType());

        //设置时间
        vmCreateModel.setCreateTime(new Date());
        //设置用户uid,暂时固定为1
        vmCreateModel.setUserId(1);
        vmCreateModel.setUserName(privateVmCreateModel.getUserName());
        if (StringUtils.isNotEmpty(privateVmCreateModel.getAccountName())){
            vmCreateModel.setAccountName(privateVmCreateModel.getAccountName());
        }else {
            vmCreateModel.setAccountName(privateVmCreateModel.getUserName());
        }

        if (StringUtils.isNotEmpty(privateVmCreateModel.getArch())){
            vmCreateModel.setArch(privateVmCreateModel.getArch());
        }else {
            vmCreateModel.setArch("CPU");
        }

        if (StringUtils.isNotEmpty(privateVmCreateModel.getCloudScope())){
            vmCreateModel.setCloudScope(privateVmCreateModel.getCloudScope());
        }

        //用这个svcId，避免给其他服务发送消息
        vmCreateModel.setSvcId(CnapCloudMgmtConstants.SCALE_SVC_PRE+ UUID.randomUUID().toString().substring(0,8));
        vmCreateModel.setAppId(vmCreateModel.getSvcId());

        vmCreateModel.setCategory("ai");

        //设置虚机的东西
        vmCreateModel.setVmModels(new ArrayList<>());
        VmModel vmModel = new VmModel();
        vmModel.setName(privateVmCreateModel.getVmName());
        vmModel.setUserName(privateVmCreateModel.getUserName());
        vmModel.setCloudType(CloudVendorsEnum.getCloudeTypeByDescribe(privateVmCreateModel.getCloudVendor()));
        if (StringUtils.isNotBlank(privateVmCreateModel.getExtranlIp())) {
            vmModel.setExternalIp(privateVmCreateModel.getExtranlIp());
        }else{
            vmModel.setExternalIp(privateVmCreateModel.getInnerIp());
        }
        vmModel.setInnerIp(privateVmCreateModel.getInnerIp());
        if (StringUtils.isNotBlank(privateVmCreateModel.getRegionId())) {
            vmModel.setRegionId(UUID.randomUUID().toString().substring(0,4) +
                    privateVmCreateModel.getRegionId());
        }else {
            vmModel.setRegionId(UUID.randomUUID().toString().substring(0,8));
        }
        //zoneId特殊处理，为了防止扩容的时候被选择
        if (StringUtils.isNotBlank(privateVmCreateModel.getZoneId())) {
            vmModel.setZoneId(UUID.randomUUID().toString().substring(0,4) + privateVmCreateModel.getZoneId());
        }else {
            vmModel.setZoneId(UUID.randomUUID().toString().substring(0,8));

        }
        vmModel.setPasswd(privateVmCreateModel.getPasswd());
        vmModel.setLoginUser("root");
        vmModel.setOsPlatform(privateVmCreateModel.getOsVersion());
        vmModel.setAddclusterName(privateVmCreateModel.getK8sClusterName());
        vmModel.setGpuFlavor(privateVmCreateModel.getGpuFlavor());
        vmModel.setCategory("ai");
        if (StringUtils.isNotEmpty(privateVmCreateModel.getBusinessId())) {
            vmModel.setBusinessId(privateVmCreateModel.getBusinessId());
        }else {
            vmModel.setBusinessId(CnapCloudMgmtConstants.VM_FROM_INPUT + UUID.randomUUID().toString().substring(0,8));
        }
        vmCreateModel.getVmModels().add(vmModel);
        vmCreateModel.setWaitSvcId(CnapCloudMgmtConstants.DEFAULT_WAIT_APP_ID);

        vmCreateInfoMapper.saveVmCreateRecord(vmCreateModel);
    }

    /**
     * 做成异步的
     * 保存到数据库，等定时任务调用再执行
     *
     * @param vmCreateModel 资源的需求
     */
    @Override
    public void createVmResource(VmCreateModel vmCreateModel) {
        LOGGER.warn("createVmMachines param={}",JSONObject.toJSONString(vmCreateModel));


        VmCreateModel oldVmCreateModel
                = vmCreateInfoMapper.getCreateInfoBySvcId(vmCreateModel.getSvcId());
        if (oldVmCreateModel != null) {

            cacheService.removeStopSvcId(vmCreateModel.getSvcId());


            //说明是重试，获取当前这一步重试的步骤
            StageDetailTypeEnum retryType = StageDetailTypeEnum.getReTryStep(oldVmCreateModel);
            //如果是从0开始重试
            if (retryType.equals(StageDetailTypeEnum.ZERO)){
                oldVmCreateModel.setType(StageDetailTypeEnum.ZERO.getParentType());
                oldVmCreateModel.setStepOrder(StageDetailTypeEnum.ZERO.getStepOrder());
                oldVmCreateModel.setOrder(StageDetailTypeEnum.ZERO.getOrder());
                oldVmCreateModel.setDetailType(StageDetailTypeEnum.ZERO.getDetailType());
                oldVmCreateModel.setErrorMsg(null);
                oldVmCreateModel.setVmModelsJson(oldVmCreateModel.getCreateVmParam());
            }

            //如果是port端口打开失败，目前不会出现这个问题

            //如果安装过程中失败
            //如果3-1，说明虚机能ping通，那就继续进行
            //要把3-2 是copy，防止拷贝到一半有异常，也进行重新拉取
            if (retryType.equals(StageDetailTypeEnum.THREE_ONE)){

                oldVmCreateModel.setType(StageDetailTypeEnum.THREE_ONE.getParentType());
                oldVmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_ONE.getStepOrder());
                oldVmCreateModel.setErrorMsg(null);

            }

            //进行到3-3是重新解压包
            if (retryType.equals(StageDetailTypeEnum.THREE_THREE)){

                oldVmCreateModel.setType(StageDetailTypeEnum.THREE_THREE.getParentType());
                oldVmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_THREE.getStepOrder());
                oldVmCreateModel.setErrorMsg(null);

                //重新发送3-1.3-2的消息
                send3132Msg(oldVmCreateModel);
            }



            //进行到3-4是安装插件
            if (retryType.equals(StageDetailTypeEnum.THREE_FOUR)){
                oldVmCreateModel.setType(StageDetailTypeEnum.THREE_FOUR.getParentType());
                oldVmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FOUR.getStepOrder());
                oldVmCreateModel.setErrorMsg(null);

                //重新发送3-1.3-2,3-3的消息
                send3132Msg(oldVmCreateModel);
                send33Msg(oldVmCreateModel);
            }

            //进行到3-5是安装参数设置
            if (retryType.equals(StageDetailTypeEnum.THREE_FIVE)){

                oldVmCreateModel.setType(StageDetailTypeEnum.THREE_FIVE.getParentType());
                oldVmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_FIVE.getStepOrder());
                oldVmCreateModel.setErrorMsg(null);

                //重新发送3-1.3-2,3-3,3-4的消息
                send3132Msg(oldVmCreateModel);
                send33Msg(oldVmCreateModel);
                send34Msg(oldVmCreateModel);
            }

            //进行到3-6是安装
            if (retryType.equals(StageDetailTypeEnum.THREE_SIX)){

                oldVmCreateModel.setType(StageDetailTypeEnum.THREE_SIX.getParentType());
                oldVmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_SIX.getStepOrder());
                oldVmCreateModel.setErrorMsg(null);

                //重新发送3-1.3-2,3-3,3-4,3-5的消息
                send3132Msg(oldVmCreateModel);
                send33Msg(oldVmCreateModel);
                send34Msg(oldVmCreateModel);
                send35Msg(oldVmCreateModel);
            }

            //更新重试信息
            vmCreateInfoMapper.updateVmCreateInfo(oldVmCreateModel);

        } else {

            cacheService.removeStopSvcId(vmCreateModel.getSvcId());

            //设置userName和userid
            String userName = null;
            if (StringUtils.isNotEmpty(vmCreateModel.getUserName())) {
                 userName = vmCreateModel.getUserName();
            }else {
                userName = TokenUtils.getUserName();
            }
            //目前阶段写死为1
            int userId = 0;//TokenUtils.getUserId();
            if (isNumeric(vmCreateModel.getAccountName())){
                userId = Integer.parseInt(vmCreateModel.getAccountName());
            }
            LOGGER.warn("createVmResource svcId={} userId={} userName={} accountName={}",
                    vmCreateModel.getSvcId(),userId,userName,vmCreateModel.getAccountName());

            //刚开始是初始化
            vmCreateModel.setType(StageDetailTypeEnum.ZERO.getParentType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.ZERO.getStepOrder());
            vmCreateModel.setOrder(StageDetailTypeEnum.ZERO.getOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.ZERO.getDetailType());
            //设置时间
            vmCreateModel.setCreateTime(new Date());
            //设置用户uid
            vmCreateModel.setUserId(userId);
            //设置标签
            //每个vm设置userId和name
            if ((vmCreateModel.getVmModels() != null)
                    &&(!vmCreateModel.getVmModels().isEmpty())){
                for (VmModel vmModel:vmCreateModel.getVmModels()){
                    vmModel.setCategory(vmCreateModel.getCategory());
                    vmModel.setUserName(userName);
                    vmModel.setUserId(userId);
                    vmModel.setSvcNameLabelValue(vmCreateModel.getSvcName());
                    vmModel.setSvcIdLabelValue(getRealSvcId(vmCreateModel.getSvcId()));
                    vmModel.setArch(vmCreateModel.getArch());
                    vmModel.setAccountName(vmCreateModel.getAccountName());
                }
            }

            //备份一下创建参数
            vmCreateModel.setCreateVmParam(JSONObject.toJSONString(vmCreateModel.getVmModels()));
            //所有的都是wait都是待处理
            vmCreateModel.setWaitSvcId(CnapCloudMgmtConstants.DEFAULT_WAIT_APP_ID);
            //刷新一下vmCreateJson
            vmCreateModel.setVmModelsJson(null);
            LOGGER.warn("saveVmCreateRecord json={}",JSONObject.toJSONString(vmCreateModel));
            vmCreateInfoMapper.saveVmCreateRecord(vmCreateModel);
        }
    }

    private void send3132Msg(VmCreateModel vmCreateModel){

        VmCreateModel vmSendCreateModel31 = new VmCreateModel();
        vmSendCreateModel31.setType(StageTypeEnum.K8S.getType());
        vmSendCreateModel31.setStepOrder(StageDetailTypeEnum.THREE_ONE.getStepOrder());
        vmSendCreateModel31.setDetailType(StageDetailTypeEnum.THREE_ONE.getDetailType());
        vmSendCreateModel31.setOrder(StageDetailTypeEnum.THREE_ONE.getOrder());
        vmSendCreateModel31.setSvcId(vmCreateModel.getSvcId());
        updateStageMsg(vmSendCreateModel31);

        VmCreateModel vmSendCreateModel32 = new VmCreateModel();
        vmSendCreateModel32.setType(StageTypeEnum.K8S.getType());
        vmSendCreateModel32.setStepOrder(StageDetailTypeEnum.THREE_TOW.getStepOrder());
        vmSendCreateModel32.setDetailType(StageDetailTypeEnum.THREE_TOW.getDetailType());
        vmSendCreateModel32.setOrder(StageDetailTypeEnum.THREE_TOW.getOrder());
        vmSendCreateModel32.setSvcId(vmCreateModel.getSvcId());
        updateStageMsg(vmSendCreateModel31);
    }

    private void send33Msg(VmCreateModel vmCreateModel){

        VmCreateModel vmSendCreateModel33 = new VmCreateModel();
        vmSendCreateModel33.setType(StageTypeEnum.K8S.getType());
        vmSendCreateModel33.setStepOrder(StageDetailTypeEnum.THREE_THREE.getStepOrder());
        vmSendCreateModel33.setDetailType(StageDetailTypeEnum.THREE_THREE.getDetailType());
        vmSendCreateModel33.setOrder(StageDetailTypeEnum.THREE_THREE.getOrder());
        vmSendCreateModel33.setSvcId(vmCreateModel.getSvcId());
        updateStageMsg(vmSendCreateModel33);

    }

    private void send34Msg(VmCreateModel vmCreateModel){

        VmCreateModel vmSendCreateModel34 = new VmCreateModel();
        vmSendCreateModel34.setType(StageTypeEnum.K8S.getType());
        vmSendCreateModel34.setStepOrder(StageDetailTypeEnum.THREE_FOUR.getStepOrder());
        vmSendCreateModel34.setDetailType(StageDetailTypeEnum.THREE_FOUR.getDetailType());
        vmSendCreateModel34.setOrder(StageDetailTypeEnum.THREE_FOUR.getOrder());
        vmSendCreateModel34.setSvcId(vmCreateModel.getSvcId());
        updateStageMsg(vmSendCreateModel34);

    }

    private void send35Msg(VmCreateModel vmCreateModel){

        VmCreateModel vmSendCreateModel35 = new VmCreateModel();
        vmSendCreateModel35.setType(StageTypeEnum.K8S.getType());
        vmSendCreateModel35.setStepOrder(StageDetailTypeEnum.THREE_FIVE.getStepOrder());
        vmSendCreateModel35.setDetailType(StageDetailTypeEnum.THREE_FIVE.getDetailType());
        vmSendCreateModel35.setOrder(StageDetailTypeEnum.THREE_FIVE.getOrder());
        vmSendCreateModel35.setSvcId(vmCreateModel.getSvcId());
        updateStageMsg(vmSendCreateModel35);

    }

    /**
     * svcId不能重复
     * 所以新建id保存
     * @param vmCreateModel
     */
    @Override
    public void endVmCreateInfo(VmCreateModel vmCreateModel) {
        String time = DateUtils.changeDateFormatYYYYMMDDHHMM(new Date());
        //APP-id也设置成新的
        String newAppId = vmCreateModel.getAppId() + time;
        //删除锁
        cacheService.delVmCreate(vmCreateModel.getSvcId());
        //防止svc重复
        String newSvcId = vmCreateModel.getSvcId() + time;
        //结束
        vmCreateInfoMapper.endVmCreateInfo(vmCreateModel.getSvcId(),newSvcId,newAppId,StageTypeEnum.END.getType());
    }

    /**
     *      * 开通云资源，完成以下内容
     *      * 1.申请开通虚机，调用其他接口
     *      * 2.完成安全组开通，
     *      * 3.完成k8s等组件安装
     *      * 4、完成调度平台组件安装
     *      * 如果svcId已经存在了，且errorMsg不为空，那就重试
     *      完成安装之后会同步harbor
     * @param  vmCreateModel
     */
    @Async("cnapTaskExcutor")  //执行多线程异步任务
    @Override
    public void handleVmCreateInfo(VmCreateModel vmCreateModel) {

        try {

            //如果为空，说明本次不需要创建主机
            if ((vmCreateModel.getVmModels() == null) || (vmCreateModel.getVmModels().isEmpty())) {
                handleEmptyVmModels(vmCreateModel);
                return;
            }

            //进行虚机创建等操作
            handleVmModels(vmCreateModel);

        } catch (Throwable e) {
            LOGGER.error("handleVmCreateInfo error svcId={} type={}", vmCreateModel.getSvcId(), vmCreateModel.getType(), e);
        }
    }

    /**
     *
     * @param vmCreateModel
     */
    @Override
    public void handleVmModels(VmCreateModel vmCreateModel) {

        //开始创建
        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("handleVmModels begin svcId={} ", vmCreateModel.getSvcId());

        //如果之前异常了，就不在进行处理
        if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())){
            //删除锁
            cacheService.delVmCreate(vmCreateModel.getSvcId());
            return;
        }

        //第一步创建机器
        if (StringUtils.equals(vmCreateModel.getType(), StageTypeEnum.INIT.getType())) {

            //防止多线程异常
            if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())
                    ||StringUtils.isNotEmpty(vmCreateInfoMapper.getErrorMsg(vmCreateModel.getSvcId()))){
                //删除锁
                cacheService.delVmCreate(vmCreateModel.getSvcId());
                return;
            }

            //开始创建虚机
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("handleVmModels createNewVmMachines begin svcId={} ", vmCreateModel.getSvcId());

            //看看是不是进行途中，删除了微服务，那就要取消创建
            if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
                removeCreateVmProcess(getRealSvcId(vmCreateModel.getSvcId()));
                return;
            }

            //创建新的机器
            createNewVmMachines(vmCreateModel);
        }


        //看看是不是进行途中，删除了微服务，那就要取消创建
        if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){

            //删除机器
            if (StringUtils.isNotEmpty(vmCreateModel.getVmModels().get(0).getPasswd())) {
                delVms(vmCreateModel.getVmModels());
            }
            removeCreateVmProcess(getRealSvcId(vmCreateModel.getSvcId()));
            return;
        }

        //第二步开通端口
        if (StringUtils.equals(vmCreateModel.getType(), StageTypeEnum.CLOUD_RESOURCE.getType())) {

            //防止多线程异常
            if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())
                    ||StringUtils.isNotEmpty(vmCreateInfoMapper.getErrorMsg(vmCreateModel.getSvcId()))){
                cacheService.delVmCreate(vmCreateModel.getSvcId());
                return;
            }

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("handleVmModels openVmPort begin svcId={} ", vmCreateModel.getSvcId());
            openVmPort(vmCreateModel);
        }

        //看看是不是进行途中，删除了微服务，那就要取消创建
        if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
            //删除机器
            if (StringUtils.isNotEmpty(vmCreateModel.getVmModels().get(0).getPasswd())) {
                delVms(vmCreateModel.getVmModels());
            }
            removeCreateVmProcess(getRealSvcId(vmCreateModel.getSvcId()));
            return;
        }

        //第三步安装各种框架，如果是重试，有可能直接进入这一步
        if (StringUtils.equals(vmCreateModel.getType(), StageTypeEnum.SAFE_PORT.getType())
               ||StringUtils.equals(vmCreateModel.getType(),StageTypeEnum.K8S.getType())){

            //防止多线程异常
            if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())){
                cacheService.delVmCreate(vmCreateModel.getSvcId());
                return;
            }

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("handleVmModels installK8s begin svcId={} ", vmCreateModel.getSvcId());
            installK8s(vmCreateModel);
        }

        //看看是不是进行途中，删除了微服务，那就要取消创建
        if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){

            //删除机器在install过程中去做
            removeCreateVmProcess(getRealSvcId(vmCreateModel.getSvcId()));
            return;
        }


        //确保删除锁
        cacheService.delVmCreate(vmCreateModel.getSvcId());
    }

    //安装k8s相关的东西
    private void installK8s(VmCreateModel vmCreateModel) {

        //防止多线程异常
        if (StringUtils.isEmpty(vmCreateModel.getVmModels().get(0).getPasswd())){
            vmCreateModel.setErrorMsg("vm no passwd");
            //发送通知
            updateStageMsg(vmCreateModel);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            return;
        }

        String k8sClusterName = null;
        String cloudeVendor = null;
        try{

            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("installK8s begin svcId={}",vmCreateModel.getSvcId());
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");

            //真正的安装，等待主机能ping通才行
            //如果步骤大于3-1的步骤，才进行ping，否则就是重试，要跳过这一步
            if (StageDetailTypeEnum.THREE_ONE.getStepOrder() > vmCreateModel.getStepOrder()) {

                //看看是不是进行途中，删除了微服务，那就要取消创建
                if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
                    return;
                }

                int i = 0;
                while (true) {

                    boolean pingResult = true;
                    for (VmModel vmModel : vmCreateModel.getVmModels()) {
                        boolean tmpPingResult = cnapSshService.checkVmNetConnectSuccess(vmModel);
                        //都为true才是true
                        if ((pingResult) && (!tmpPingResult)) {
                            pingResult = tmpPingResult;
                        }
                        LOGGER.warn("installK8s check ping result={} targetIp={}",
                                tmpPingResult, vmModel.getExternalIp());
                    }

                    if (pingResult) {

                        //到这一步，说明云资源已经ping通了，
                        //改状态，通知其他资源
                        vmCreateModel.setType(StageTypeEnum.K8S.getType());
                        vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_ONE.getStepOrder());
                        vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_ONE.getDetailType());
                        vmCreateModel.setOrder(StageDetailTypeEnum.THREE_ONE.getOrder());

                        VmCreateModel vmSendCreateModel31 = new VmCreateModel();
                        initVmSendCreateModel(vmSendCreateModel31, vmCreateModel);
                        updateStageMsg(vmSendCreateModel31);
                        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
                        break;
                    }

                    i++;
                    if (i > 10) {
                        throw new Exception("wait host ls al" + vmCreateModel.getVmModels().get(0).getExternalIp()
                                + " too many times" + i);
                    }
                    try {
                        Thread.sleep(30 * 1000L);
                    } catch (InterruptedException e) {
                        //防止线程异常，暂停失败
                        LOGGER.error("installK8s check ping sleep fail e", e);
                        Thread.currentThread().interrupt();
                    } catch (Exception e) {
                        LOGGER.error("installK8s check ping sleep err. ", e);
                    }
                }
            }

            //看看是不是进行途中，删除了微服务，那就要取消创建
            if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
                return;
            }

            //安装k8s
            String k8sClusterIp = multiCloudMgmtService.installK8sFramwork(vmCreateModel);


            //k8sClusterIp为null，就说明是扩容
            if (StringUtils.isNotEmpty(k8sClusterIp)) {

                //看看是不是进行途中，删除了微服务，那就要取消创建
                if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
                    multiCloudMgmtService.delK8sCluster(k8sClusterIp);
                    return;
                }

                LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
                LOGGER.warn("installCnapScheduleAgent begin k8sClusterIp={}",k8sClusterIp);
                LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");

                //安装agent
                K8sClusterBasicModel k8sClusterBasicModel =
                        k8sClusterService.installCnapScheduleAgent(k8sClusterIp,vmCreateModel.getArch());
                //暂停几秒，防止通知过早，agent还没有启动
                try{
                    Thread.sleep(15*1000L);
                } catch(InterruptedException e){
                    //防止线程异常，暂停失败
                    LOGGER.error("after installCnapScheduleAgent sleep fail e",e);
                    Thread.currentThread().interrupt();
                }catch (Exception e) {
                    LOGGER.error("after installCnapScheduleAgent sleep fail err. ",e);
                }

                //如果是集群间扩容，那就不执行调度策略的更新
                if (StringUtils.containsIgnoreCase(vmCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)) {

                    //通知调度引擎执行部署应用
                    multiCloudMgmtService.informOtherSvcClusterChange(k8sClusterBasicModel.getK8sClusterName());

                }

                //到这一步，说明开始镜像拉取
                //改状态，通知其他资源
                vmCreateModel.setType(StageTypeEnum.K8S.getType());
                vmCreateModel.setStepOrder(StageDetailTypeEnum.THREE_SEVEN.getStepOrder());
                vmCreateModel.setDetailType(StageDetailTypeEnum.THREE_SEVEN.getDetailType());
                vmCreateModel.setOrder(StageDetailTypeEnum.THREE_SEVEN.getOrder());

                if (!StringUtils.equalsIgnoreCase(vmCreateModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)) {
                    VmCreateModel vmSendCreateModel37 = new VmCreateModel();
                    List<Map<String, String>> extranInfo = new ArrayList<>();
                    for (VmModel vmModel : vmCreateModel.getVmModels()) {
                        Map<String, String> param = new HashMap<>();
                        param.put("name", k8sClusterBasicModel.getK8sClusterName());
                        param.put("providerName", CloudVendorsEnum.getCloudeTypeByDescribe(k8sClusterBasicModel.getCloudVendor()));
                        param.put("ip", vmModel.getExternalIp());
                        param.put("innerIp", vmModel.getInnerIp());
                        extranInfo.add(param);
                    }
                    vmSendCreateModel37.setExtraInfo(extranInfo);
                    initVmSendCreateModel(vmSendCreateModel37, vmCreateModel);
                    updateStageMsg(vmSendCreateModel37);
                }


                //如果是gpu还要通知几部
                if (StringUtils.equalsIgnoreCase(vmCreateModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){

                    VmCreateModel vmSendCreateModelGpu37 = new VmCreateModel();
                    vmCreateModel.setType(StageTypeEnum.K8S.getType());
                    vmCreateModel.setStepOrder(StageEnum2.THREE_SEVEN_PUBLIC_GPU.getOrder());
                    vmCreateModel.setDetailType(StageEnum2.THREE_SEVEN_PUBLIC_GPU.getDetailType());
                    vmCreateModel.setOrder(StageEnum2.THREE_SEVEN_PUBLIC_GPU.getOrder());
                    initVmSendCreateModel(vmSendCreateModelGpu37,vmCreateModel);
                    updateStageMsg(vmSendCreateModelGpu37);

                    VmCreateModel vmSendCreateModelGpu38 = new VmCreateModel();
                    vmCreateModel.setType(StageTypeEnum.K8S.getType());
                    vmCreateModel.setStepOrder(StageEnum2.THREE_EIGHT_PUBLIC_GPU.getOrder());
                    vmCreateModel.setDetailType(StageEnum2.THREE_EIGHT_PUBLIC_GPU.getDetailType());
                    vmCreateModel.setOrder(StageEnum2.THREE_EIGHT_PUBLIC_GPU.getOrder());
                    initVmSendCreateModel(vmSendCreateModelGpu38,vmCreateModel);
                    updateStageMsg(vmSendCreateModelGpu38);


                    VmCreateModel vmSendCreateModelGpu39 = new VmCreateModel();
                    vmCreateModel.setType(StageTypeEnum.K8S.getType());
                    vmCreateModel.setStepOrder(StageEnum2.THREE_NINE_PUBLIC_GPU.getOrder());
                    vmCreateModel.setDetailType(StageEnum2.THREE_NINE_PUBLIC_GPU.getDetailType());
                    vmCreateModel.setOrder(StageEnum2.THREE_NINE_PUBLIC_GPU.getOrder());
                    List<Map<String,String>> extranInfo = new ArrayList<>();
                    for (VmModel vmModel:vmCreateModel.getVmModels()){
                        Map<String,String> param = new HashMap<>();
                        param.put("name",k8sClusterBasicModel.getK8sClusterName());
                        param.put("providerName",CloudVendorsEnum.getCloudeTypeByDescribe(k8sClusterBasicModel.getCloudVendor()));
                        param.put("ip",vmModel.getExternalIp());
                        param.put("innerIp",vmModel.getInnerIp());
                        extranInfo.add(param);
                    }
                    vmSendCreateModelGpu39.setExtraInfo(extranInfo);
                    initVmSendCreateModel(vmSendCreateModelGpu39,vmCreateModel);
                    updateStageMsg(vmSendCreateModelGpu39);

                }

                vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
                vmCreateInfoMapper.updateWaitCreateInfo0(vmCreateModel.getSvcId());
                //结束通知

                //应用迁移需要干点事情
                if (StringUtils.contains(vmCreateModel.getSvcId(),CnapCloudMgmtConstants.MIGRATE_VM)){

                    NotifyClusterInfoDTO notifyClusterInfoDTO = new NotifyClusterInfoDTO();
                    notifyClusterInfoDTO.setSvcId(vmCreateModel.getSvcId().split("-")[1]);
                    notifyClusterInfoDTO.setClusterName(k8sClusterBasicModel.getK8sClusterName());
                    notifyClusterInfoDTO.setProviderName(
                            CloudVendorsEnum.getCloudeTypeByDescribe(k8sClusterBasicModel.getCloudVendor()));
                    notifyClusterInfoDTO.setIp(k8sClusterBasicModel.getK8sClusterIp());
                    notifyClusterInfoDTO.setInnerIp(k8sClusterBasicModel.getMasterNodeInnerIp());
                    notifyClusterMigrate(notifyClusterInfoDTO);
                }

                /**
                 * 完成整个流程，设置流程为结束
                 */
                vmCreateModel.setType(StageTypeEnum.END.getType());
                vmCreateModel.setStepOrder(StageDetailTypeEnum.END.getStepOrder());
                vmCreateModel.setDetailType(StageDetailTypeEnum.END.getDetailType());
                vmCreateModel.setOrder(StageDetailTypeEnum.END.getOrder());

            }else{

                //这里说明之前有过一个k8s集群，他成为了那个集群的worker节点
                VmModel firstVmModel = vmCreateModel.getVmModels().get(0);
                List<K8sClusterModel> k8sClusterModels = null;

                //for ut
                if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,vmCreateModel.getSvcId())){
                    k8sClusterModels = new ArrayList<>();
                    K8sClusterModel k8sClusterModel = new K8sClusterModel();
                    k8sClusterModels.add(k8sClusterModel);
                    k8sClusterModel.setVmModels(new ArrayList<>());
                    VmModel vmModel = new VmModel();
                    k8sClusterModel.getVmModels().add(vmModel);
                }else {

                    //获取集群信息
                    k8sClusterModels =
                            multiCloudMgmtService.getK8sClusterInfosByZone(firstVmModel, vmCreateModel.getUserName());
                }
                LOGGER.warn("k8s exist begin to find cluster info svcId={}",vmCreateModel.getSvcId());
                for (K8sClusterModel k8sClusterModelExist:k8sClusterModels){

                    //看看是不是进行途中，删除了微服务，那就要取消创建
                    if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
                        delVms(vmCreateModel.getVmModels());
                        return;
                    }

                    for (VmModel vmModel:k8sClusterModelExist.getVmModels()){
                        if (StringUtils.equals(vmModel.getVmId(),firstVmModel.getVmId())){

                            k8sClusterName = k8sClusterModelExist.getK8sClusterName();
                            cloudeVendor = k8sClusterModelExist.getCloudVendor();

                            LOGGER.warn("k8s exist find result svcId={} vmId={} clustername={}",vmCreateModel.getSvcId(),
                                    vmModel.getVmId(),k8sClusterName);
                            //去通知一下策略扩容
                            //要看扩容是手动扩的还是自动，自动扩容不需要改策略

                            //如果是集群间扩容，那就不执行调度策略的更新
                            if (StringUtils.containsIgnoreCase(vmCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)) {

                                //通知调度引擎执行部署
                                multiCloudMgmtService.informOtherSvcClusterChange(k8sClusterModelExist.getK8sClusterName());
                            }

                            //应用迁移需要干点事情
                            if (StringUtils.contains(vmCreateModel.getSvcId(),CnapCloudMgmtConstants.MIGRATE_VM)){
                                NotifyClusterInfoDTO notifyClusterInfoDTO = new NotifyClusterInfoDTO();
                                notifyClusterInfoDTO.setSvcId(vmCreateModel.getSvcId().split("-")[1]);
                                notifyClusterInfoDTO.setClusterName(k8sClusterModelExist.getK8sClusterName());
                                notifyClusterInfoDTO.setProviderName(
                                        CloudVendorsEnum.getCloudeTypeByDescribe(k8sClusterModelExist.getCloudVendor()));
                                notifyClusterInfoDTO.setIp(k8sClusterModelExist.getK8sClusterIp());
                                notifyClusterInfoDTO.setInnerIp(k8sClusterModelExist.getVmModels().get(0).getInnerIp());
                                notifyClusterMigrate(notifyClusterInfoDTO);
                            }
                        }
                    }
                }

                if (StringUtils.isEmpty(k8sClusterName)||StringUtils.isEmpty(cloudeVendor)){
                    throw new Exception("add new worker fail svcId="+vmCreateModel.getSvcId());
                }
            }

        } catch(InterruptedException e){
            //防止线程异常，暂停失败
            LOGGER.error("installK8s fail error  sleep fail e",e);
            Thread.currentThread().interrupt();
            vmCreateModel.setErrorMsg(e.getMessage());

        }catch (Throwable e) {

            LOGGER.error("installK8s fail error svcId={}",vmCreateModel.getSvcId(),e);
            vmCreateModel.setErrorMsg(e.getMessage());
        }
        
        //更新状态
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())){
            cacheService.delVmCreate(vmCreateModel.getSvcId());
            updateStageMsg(vmCreateModel);
        }

        //完成整个安装流程
        if (StringUtils.isEmpty(vmCreateModel.getErrorMsg())) {
            endVmCreateInfo(vmCreateModel);
        }
    }

    private void openVmPort(VmCreateModel vmCreateModel) {

        //防止多线程异常
        if (StringUtils.isEmpty(vmCreateModel.getVmModels().get(0).getPasswd())){
            vmCreateModel.setErrorMsg("vm no passwd");
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
            //发送通知
            updateStageMsg(vmCreateModel);
            cacheService.delVmCreate(vmCreateModel.getSvcId());
            return;
        }

        //先把状态更新成创建中
        //vmCreateInfoMapper.updateType(vmCreateModel.getSvcId(), CreateVmStepEnum.PortCreate.getDescribe());

        try{
        //todo 去其他服务开端口,当前先不开，默认所有打开
            vmCreateModel.setType(StageTypeEnum.SAFE_PORT.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.TOW_ONE.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.TOW_ONE.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.TOW_ONE.getOrder());

            VmCreateModel vmSendCreateModelTwoOne = new VmCreateModel();
            initVmSendCreateModel(vmSendCreateModelTwoOne,vmCreateModel);
            updateStageMsg(vmSendCreateModelTwoOne);

            //申请
            vmCreateModel.setType(StageTypeEnum.SAFE_PORT.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.TOW_TOW.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.TOW_TOW.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.TOW_TOW.getOrder());

            VmCreateModel vmSendCreateModelTwoTwo = new VmCreateModel();
            initVmSendCreateModel(vmSendCreateModelTwoTwo,vmCreateModel);
            updateStageMsg(vmSendCreateModelTwoTwo);

            //等待
            vmCreateModel.setType(StageTypeEnum.SAFE_PORT.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.TOW_THREE.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.TOW_THREE.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.TOW_THREE.getOrder());

            VmCreateModel vmSendCreateModelTwoThree = new VmCreateModel();
            initVmSendCreateModel(vmSendCreateModelTwoThree,vmCreateModel);
            updateStageMsg(vmSendCreateModelTwoThree);

            //完成端口开通
            vmCreateModel.setType(StageTypeEnum.SAFE_PORT.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.TOW_FOUR.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.TOW_FOUR.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.TOW_FOUR.getOrder());

            VmCreateModel vmSendCreateModelTwoFour = new VmCreateModel();
            initVmSendCreateModel(vmSendCreateModelTwoFour,vmCreateModel);
            updateStageMsg(vmSendCreateModelTwoFour);


        }catch (Throwable e){
            LOGGER.error("openVmPort fail error svcId={}",vmCreateModel.getSvcId(),e);
            vmCreateModel.setErrorMsg(e.getMessage());
        }

        //把状态改成更新成功-或者失败
        //vmCreateModel.setType(CreateVmStepEnum.Port.getDescribe());
        //vmCreateModel.setVmModelsJson(JSONObject.toJSONString(vmCreateModel.getVmModels()));
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())){
            cacheService.delVmCreate(vmCreateModel.getSvcId());
        }

        //发送通知
        //updateStageMsg(vmCreateModel,null,null);
    }

    private void createNewVmMachines(VmCreateModel vmCreateModel) {
        //errorMsg不为空表示失败
        if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())){
            cacheService.delVmCreate(vmCreateModel.getSvcId());
            return;
        }

        try {

            //对接创建
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("createVmMachines begin svcId={} ", vmCreateModel.getSvcId());

            //第一步开始
            vmCreateModel.setType(StageTypeEnum.CLOUD_RESOURCE.getType());
            vmCreateModel.setStepOrder(StageDetailTypeEnum.ONE_ONE.getStepOrder());
            vmCreateModel.setDetailType(StageDetailTypeEnum.ONE_ONE.getDetailType());
            vmCreateModel.setOrder(StageDetailTypeEnum.ONE_ONE.getOrder());

            createVmMachines(vmCreateModel);
            
        }catch (Throwable e){
            LOGGER.error("createVmMachines fail error svcId={}",vmCreateModel.getSvcId(),e);
            vmCreateModel.setErrorMsg(e.getMessage());
        }


        //看看是不是进行途中，删除了微服务，那就要取消创建
        if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
            return;
        }

        //更新状态
        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        /**
         * 如果异常
         * 删除多线程说，通知其他系统
         */
        if (StringUtils.isNotEmpty(vmCreateModel.getErrorMsg())){
            cacheService.delVmCreate(vmCreateModel.getSvcId());
            //通知异常
            updateStageMsg(vmCreateModel);
            vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
        }
    }

    /**
     * {
     * "code": 200,
     * "dateTime": "2022-08-01T15:21:17.759+08:00[Asia/Shanghai]",
     * "errorMsg": "",
     * "result": {
     * "status": "Running"
     * },
     * "success": true
     * }
     * {
     * "businessId": "NULL/xxxxxxx",
     * "userId": 1,
     * "createVmInfo": [
     * {参数 父节点 数据类型 示例值 说明
     * code Integer 200 成功
     * dateTime String 接口响应时间
     * errorMsg String 错误信息
     * success boolean true 是否成功
     * result Object 返回数据
     * orderId result String or-aaaaaaaa 订单ID，用于回调或查询
     * 返回参数：
     * "name": "vm1",
     * "flavorId": "in-xdahdwisioo",
     * "imageId": "img-sadhwixkaoow",
     * "regionId": "cn-beijing",
     * "zoneId": "cn-beijing1",
     * "sysDiskSize": 100,
     * "dataDisks": [
     * {
     * "dataDiskType": "ssd",
     * "dataDiskSize": 50
     * }
     * ],
     * "bandwidth": 100,
     * "vmChargeType": "虚拟机付费类型(PostPaid-按需计费，PrePaid - 包年包月)",
     * "chargeUnit": "Month",
     * "chargeCount": 12,
     * "eipChargeType": "弹性IP付费类型(PostPaid-按需计费，PrePaid - 包年包月)",
     * "cloudType": "Aliyun"
     * },
     * {
     * "name": "vm2",
     * "flavorId": "in-xdahdwisioo",
     * "imageId": "img-sadhwixkaoow",
     * "regionId": "cn-beijing",
     * "zoneId": "cn-beijing1",
     * "sysDisk": 100,
     * "dataDisks": [
     * {
     * "dataDiskType": "ssd",
     * "dataDiskSize": 50
     * }
     * ],
     * "bandwidth": 100,
     * "vmChargeType": "虚拟机付费类型(PostPaid-按需计费，PrePaid - 包年包月)",
     * "chargeUnit": "Month",
     * "chargeCount": 12,
     * "eipChargeType": "弹性IP付费类型(PostPaid-按需计费，PrePaid - 包年包月)",
     * "cloudType": "Aliyun"
     * }
     * ]
     * }
     * @param vmCreateModel
     * @return
     * @throws Exception
     */
    private void createVmMachines(VmCreateModel vmCreateModel) throws Exception {

        //先检查bizId
        String bizId = vmCreateModel.getVmModels().get(0).getBusinessId();
        int userId = vmCreateModel.getUserId();
        LOGGER.warn("createVmMachines step 01 bizId={} userId={} userName={}",
                bizId,userId,vmCreateModel.getUserName());
        if (StringUtils.isEmpty(bizId)){
            //检查是扩容还是新建
            //因为没有设置bizId的集群会没法扩容，所以在这找bizId不带有vmInput的，第一个
            K8sClusterModel k8sClusterModel
                    = multiCloudMgmtService.getK8sClusterModelByZoneIdNoBizId(vmCreateModel);
            //如果强制隔离，那就新建
            if (vmCreateModel.isForcedIsolation()){
                k8sClusterModel = null;
            }

            //扩容要获取bizid
            if (k8sClusterModel != null){
                //根据zoneId拿到集群
                bizId= k8sClusterModel.getVmModels().get(0).getBusinessId();
                //从集群中拿到userId，
                userId = k8sClusterModel.getVmModels().get(0).getUserId();
                //给要扩容的机器增加bizid和uid
                for (VmModel vmModel:vmCreateModel.getVmModels()){
                    vmModel.setBusinessId(bizId);
                    vmModel.setUserId(userId);
                }
                vmCreateModel.setUserId(userId);
            }
        }
        LOGGER.warn("createVmMachines step 02 bizId={} userId={}",bizId,userId);

        //如果是扩容，那就要判断有没有有没有配额，配额没了就要抛出异常

        if (!checkUserK8sClusterQuota(vmCreateModel.getUserName())) {

            throw new Exception("配额超出限制");
        }

        //新建的虚机要生成一个随机名字
        Map<String,VmModel> vmModelMap = new HashMap<>();
        for (VmModel vmModel:vmCreateModel.getVmModels()){
            //设置虚机名字
            vmModel.setName("v" + UUID.randomUUID().toString().substring(0,8));
            vmModel.setUserName(vmCreateModel.getUserName());
            vmModelMap.put(vmModel.getName(),vmModel);
        }

        //组装请求参数
        CreateVmSendModel createVmSendModel = new CreateVmSendModel();
        createVmSendModel.setBusinessId(bizId);
        createVmSendModel.setUserId(userId);
        createVmSendModel.setCreateVmInfo(vmCreateModel.getVmModels());
        createVmSendModel.setAccountName(vmCreateModel.getAccountName());
        createVmSendModel.setUserName(vmCreateModel.getUserName());
        String createParam = JSON.toJSONString(createVmSendModel);
        //打印请求参数用于防止异常
        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("createVmMachines creat param _______:" + createParam);

        //去请求虚拟机
        String createResult
                = doHttpPostReq(VM_CONTROLER_URL + "/vm/v1/create", new StringEntity(createParam,"UTF-8"));
        CreateVmReturnModel createVmReturnModel = new CreateVmReturnModel();

        //给ut代码用
        if (StringUtils.equals(CnapCloudMgmtConstants.TEST_ADMIN,vmCreateModel.getSvcId())){
            //创建返回的参数
            CreateVmReturnDataModel createVmReturnDataModel = new CreateVmReturnDataModel();
            createVmReturnDataModel.setSuccess(true);
            createVmReturnDataModel.setOrderId(CnapCloudMgmtConstants.TEST_ADMIN);
            createVmReturnModel.setData(createVmReturnDataModel);
            createResult = JSONObject.toJSONString(createVmReturnModel);
        }
        //ut end

        //解析返回
        if (StringUtils.isNotEmpty(createResult)){
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("create vm createResult svcId={} result={}",vmCreateModel.getSvcId(),
                    createResult);

            createVmReturnModel = JSONObject.parseObject(createResult, CreateVmReturnModel.class);

        }else {
            //返回为空则异常
            throw new Exception("create vm fail bussinessId=" + createVmSendModel.getBusinessId()
                                        + " svcId="+vmCreateModel.getSvcId());
        }
        //状态不为success，则说明异常
        if (!createVmReturnModel.getData().isSuccess()){
            LOGGER.error("create vm order fail bizId={} msg={}",
                    createVmSendModel.getBusinessId(),createVmReturnModel.getData().getErrorMsg());
            throw new Exception("create vm order fail");
        }

        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("create vm order send svcId={} orderId={}",vmCreateModel.getSvcId(),
                createVmReturnModel.getData().getOrderId());
        //记录orderId
        vmCreateModel.setOrderId(createVmReturnModel.getData().getOrderId());

        /**
         * 执行到这一步，说明已经申请了资源等待开通
         * 修改自己的状态步骤
         * 通知其他服务
         */
        vmCreateModel.setType(StageTypeEnum.CLOUD_RESOURCE.getType());
        vmCreateModel.setStepOrder(StageDetailTypeEnum.ONE_ONE.getStepOrder());
        vmCreateModel.setDetailType(StageDetailTypeEnum.ONE_ONE.getDetailType());
        vmCreateModel.setOrder(StageDetailTypeEnum.ONE_ONE.getOrder());

        VmCreateModel vmSendCreateModel = new VmCreateModel();
        initVmSendCreateModel(vmSendCreateModel,vmCreateModel);
        updateStageMsg(vmSendCreateModel);
        //完成通知

        //根据orderid去查询有没有好
        int i = 0;
        int errorNum = 0;
        String errorMsg = null;
        while(true){
            //每次等待1分钟再去查询
            try{
                Thread.sleep(60 * 1000L);
            }catch(InterruptedException e){
                //防止线程异常，暂停失败
                LOGGER.error("create vm wait fail e",e);
                Thread.currentThread().interrupt();
            }catch (Exception e) {
                LOGGER.error("create vm wait fail err. ",e);
            }


            //看看是不是进行途中，删除了微服务，那就要取消创建
            if (cacheService.checkSvcIdWhetherStop(getRealSvcId(vmCreateModel.getSvcId()))){
                return;
            }

            //去检查有没有建好
            String vmResult  = doHttpPostReq(VM_CONTROLER_URL + "/vm/v1/orderQuery",
                    new StringEntity(JSON.toJSONString(createVmReturnModel.getData())));
            LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
            LOGGER.warn("query vm create return result={}", vmResult);

            //for ut
            if (StringUtils.equals(createVmReturnModel.getData().getOrderId(),
                    CnapCloudMgmtConstants.TEST_ADMIN)){
                //构造ut的结构体
                VmOrderReturnModel vmOrderReturnModel = new VmOrderReturnModel();
                VmOrderInfoModel vmOrderInfoModel = new VmOrderInfoModel();
                vmOrderReturnModel.setData(vmOrderInfoModel);
                vmOrderInfoModel.setBusinessId(CnapCloudMgmtConstants.TEST_ADMIN);
                vmOrderInfoModel.setOrderStatus("SUCCESS");
                vmOrderInfoModel.setVms(new ArrayList<>());
                VmModel vmModel = new VmModel();
                vmModel.setPasswd("passwd");
                vmOrderInfoModel.getVms().add(vmModel);
                vmModel.setName(vmCreateModel.getVmModels().get(0).getName());

                vmResult = JSONObject.toJSONString(vmOrderReturnModel);
            }
            //ut end

            if (StringUtils.isNotEmpty(vmResult)) {

                VmOrderReturnModel vmOrderReturnModel
                        = JSONObject.parseObject(vmResult, VmOrderReturnModel.class);
                if (vmOrderReturnModel.getCode() != 500) {
                    String status = "error";
                    if (vmOrderReturnModel.getData() != null){
                        status = vmOrderReturnModel.getData().getOrderStatus();
                    }

                    LOGGER.warn("query vm create svcId={} status={} oderId={}",
                            vmCreateModel.getSvcId(),
                            status, vmCreateModel.getOrderId());
                    if (StringUtils.equalsIgnoreCase(status,"error")
                            ||StringUtils.equalsIgnoreCase(status, "failed")
                            ||StringUtils.equalsIgnoreCase(status,"Failed")){
                        errorNum++;
                        if ((vmOrderReturnModel != null)&&(vmOrderReturnModel.getData() != null)){
                            errorMsg = vmOrderReturnModel.getData().getErrorMessage();
                        }
                    }

                    //如果状态是SUCCESS，表明已经创建好了
                    //状态是running，就是等着创建
                    if (StringUtils.equalsIgnoreCase(status, "SUCCESS")) {

                        for (VmModel vmModel : vmOrderReturnModel.getData().getVms()) {
                            VmModel oldVmModel = vmModelMap.get(vmModel.getName());
                            if (StringUtils.isEmpty(vmModel.getRegionId())) {
                                vmModel.setRegionId(oldVmModel.getRegionId());
                            }
                            if (StringUtils.isEmpty(vmModel.getRegionId())){
                                vmModel.setRegionId(oldVmModel.getCloudType());
                            }
                            if (StringUtils.isEmpty(vmModel.getZoneId())) {
                                vmModel.setZoneId(oldVmModel.getZoneId());
                            }
                            if (StringUtils.isEmpty(vmModel.getZoneId())){
                                vmModel.setZoneId(oldVmModel.getCloudType());
                            }
                            if (StringUtils.isEmpty(vmModel.getBusinessId())) {
                                vmModel.setBusinessId(vmOrderReturnModel.getData().getBusinessId());
                            }
                            if (StringUtils.isEmpty(vmModel.getUserName())) {
                                vmModel.setUserName(vmCreateModel.getUserName());
                            }
                            if (StringUtils.isEmpty(vmModel.getBusinessId())) {
                                vmModel.setBusinessId(bizId);
                            }
                            if (StringUtils.isEmpty(vmModel.getCategory())){
                                vmModel.setCategory(oldVmModel.getCategory());
                            }
                            if (StringUtils.isEmpty(vmModel.getSvcNameLabelValue())){
                                vmModel.setSvcNameLabelValue(oldVmModel.getSvcNameLabelValue());
                            }
                            if (StringUtils.isEmpty(vmModel.getSvcIdLabelValue())){
                                vmModel.setSvcIdLabelValue(oldVmModel.getSvcIdLabelValue());
                            }
                            if (vmModel.getUserId() == 0){
                                vmModel.setUserId(userId);
                            }

                            if (StringUtils.isEmpty(vmModel.getAccountName())){
                                vmModel.setAccountName(oldVmModel.getAccountName());
                            }
                            if (StringUtils.isEmpty(vmModel.getArch())){
                                vmModel.setArch(oldVmModel.getArch());
                            }
                            if (StringUtils.isEmpty(vmModel.getCloudType())){
                                vmModel.setCloudType(oldVmModel.getCloudType());
                            }
                            if(StringUtils.isEmpty(vmModel.getFlavorId())){
                                vmModel.setFlavorId(oldVmModel.getFlavorId());
                            }
                            //获取规格
                            if (StringUtils.endsWithIgnoreCase(vmModel.getArch(),CnapCloudMgmtConstants.ARCH_CPU)
                                    ||StringUtils.isEmpty(vmModel.getArch())) {

                                LOGGER.warn("query vm flavor account={} cloudType={} region={} zone={} flavorId={} arch={}",
                                        vmModel.getAccountName(),
                                        vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),
                                        vmModel.getFlavorId(),vmModel.getArch());
                                CloudFlavorInfoV2 cloudFlavorInfoV2
                                        = flavorInfoMapper.getFlavorWithImageDisk(vmModel.getAccountName(),
                                        vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());
                                //如果是null，那就去私有云规格查一下是不是有规格
                                if (cloudFlavorInfoV2 == null){
                                    cloudFlavorInfoV2 = flavorInfoMapper.getPrivateFlavorWithImageDisk(vmModel.getAccountName(),
                                            vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());
                                }

                                if (cloudFlavorInfoV2 == null){
                                    LOGGER.warn("query vm flavor empty account={} cloudType={} region={} zone={} flavorId={}",
                                            vmModel.getAccountName(),
                                            vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());
                                }else{

                                    //对flavor做操作
                                    Map<String,Integer> resMap = cloudFlavorInfoV2.handdleCpuAndMem();
                                    vmModel.setCpuNum(resMap.get("cpu"));
                                    vmModel.setMemByG(resMap.get("mem"));
                                }


                            }else if (StringUtils.endsWithIgnoreCase(vmModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){

                                LOGGER.warn("query GPU vm flavor account={} cloudType={} region={} zone={} flavorId={} arch={}",
                                        vmModel.getAccountName(),
                                        vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),
                                        vmModel.getFlavorId(),vmModel.getArch());
                                GpuFlavorInfo gpuFlavorInfo = flavorInfoMapper.getGpuCloudFlavorInfo(
                                        vmModel.getAccountName(),
                                        vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());

                                //为null再去查下私有云
                                if (gpuFlavorInfo == null){
                                    gpuFlavorInfo = flavorInfoMapper.getPrivateGpuCloudFlavorInfo(
                                            vmModel.getAccountName(),
                                            vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());
                                }

                                if(gpuFlavorInfo == null){
                                    LOGGER.warn("query GPU vm flavor empty account={} cloudType={} region={} zone={} flavorId={}",
                                            vmModel.getAccountName(),
                                            vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());
                                }else{
                                    FlavorGPUInfoWithImageDisk flavorGPUInfoWithImageDisk = new FlavorGPUInfoWithImageDisk();
                                    flavorGPUInfoWithImageDisk.setVal(gpuFlavorInfo.getFlavorId());
                                    flavorGPUInfoWithImageDisk.setKey(gpuFlavorInfo.getFlavorDescribe());
                                    try {
                                        flavorGPUInfoWithImageDisk.setDiskTypes(JSON.parseArray(gpuFlavorInfo.getDiskTypeStr(), DiskTypeInfo.class));
                                        flavorGPUInfoWithImageDisk.setImages(JSON.parseArray(gpuFlavorInfo.getDiskTypeStr(), ImageInfo.class));
                                        flavorGPUInfoWithImageDisk.setGpuSpec(JSON.parseObject(gpuFlavorInfo.getGpuSpecStr(), GpuSpecInfo.class));
                                        flavorGPUInfoWithImageDisk.setAccountName(gpuFlavorInfo.getAccountName());
                                        flavorGPUInfoWithImageDisk.setCloudType(gpuFlavorInfo.getCloudType());
                                        flavorGPUInfoWithImageDisk.setRegionId(gpuFlavorInfo.getRegionId());
                                        flavorGPUInfoWithImageDisk.setZoneId(gpuFlavorInfo.getZoneId());
                                    }catch (Exception e){
                                        LOGGER.error("parse gpu flavor failed account={} cloudType={} region={} zone={} flavorId={}",vmModel.getAccountName(),
                                                vmModel.getCloudType(),vmModel.getRegionId(),vmModel.getZoneId(),vmModel.getFlavorId());
                                    }

                                    vmModel.setCpuNum(flavorGPUInfoWithImageDisk.getCpu());
                                    vmModel.setMemByG(flavorGPUInfoWithImageDisk.getMem());
                                    if (flavorGPUInfoWithImageDisk.getGpuSpec() != null) {
                                        vmModel.setGpuFlavor(flavorGPUInfoWithImageDisk.getGpuSpec().getGpu() + "——" +
                                                flavorGPUInfoWithImageDisk.getGpuSpec().getGpuMemorySize());
                                    }
                                }
                            }

                            String msg = "create vm success: bizId=" + vmModel.getBusinessId() + " name=" +
                                    vmModel.getName() + "flavor info= " + vmModel.getCpuNum()+"c"+
                                    vmModel.getMemByG()+"G" + vmModel.getGpuFlavor();
                            LOGGER.warn(msg);
                            vmModel.setCnapMsg(msg);
                        }

                        vmCreateModel.setVmModelsJson(null);
                        vmCreateModel.setVmModels(vmOrderReturnModel.getData().getVms());
                        for (VmModel vmModel:vmCreateModel.getVmModels()) {
                            LOGGER.warn("create vm after save vmName={} vmSvcNameLabel={}",
                                    vmModel.getName(),vmModel.getSvcNameLabelValue());
                        }

                        /**
                         * 执行到这一步，说明虚拟机已经开通
                         * 修改自己的状态步骤
                         * 通知其他服务
                         */
                        vmCreateModel.setType(StageTypeEnum.CLOUD_RESOURCE.getType());
                        vmCreateModel.setStepOrder(StageDetailTypeEnum.ONE_TWO.getStepOrder());
                        vmCreateModel.setDetailType(StageDetailTypeEnum.ONE_TWO.getDetailType());
                        vmCreateModel.setOrder(StageDetailTypeEnum.ONE_TWO.getOrder());

                        VmCreateModel vmSendCreateModelOneTwo = new VmCreateModel();
                        initVmSendCreateModel(vmSendCreateModelOneTwo,vmCreateModel);
                        List<Map<String,String>> extranInfo = new ArrayList<>();
                        for (VmModel vmModel:vmCreateModel.getVmModels()){
                            Map<String,String> param = new HashMap<>();
                            param.put("vmName",vmModel.getName());
                            param.put("cpu",String.valueOf(vmModel.getCpuNum()));
                            param.put("memory",String.valueOf(vmModel.getMemByG()));
                            if (StringUtils.endsWithIgnoreCase(vmModel.getArch(),CnapCloudMgmtConstants.ARCH_GPU)){
                                param.put("gpuFlavor",vmModel.getGpuFlavor());
                            }
                            extranInfo.add(param);
                        }
                        vmSendCreateModelOneTwo.setExtraInfo(extranInfo);
                        updateStageMsg(vmSendCreateModelOneTwo);
                        vmCreateInfoMapper.updateVmCreateInfo(vmCreateModel);
                        //完成通知

                        break;
                    }

                }else {

                    LOGGER.warn("query vm create 500 svcId={} code={} msg={} orderId={}",
                            vmCreateModel.getSvcId(),
                            vmOrderReturnModel.getCode(),vmOrderReturnModel.getMessage(), vmCreateModel.getOrderId());
                    errorNum++;
                }
            }

            i++;
            if (i > 60){
                //最大重试60次，操作60次没建立成功，就失败
                throw new Exception("create vm too long cost "+ i);
            }
            if (errorNum > 10){
                //最多失败10次
                throw new Exception("create vm fail too many times:"+ errorMsg);

            }
        }
    }

    @Override
    public void initVmSendCreateModel(VmCreateModel vmSendCreateModel, VmCreateModel vmCreateModel) {
        vmSendCreateModel.setSvcId(vmCreateModel.getSvcId());
        vmSendCreateModel.setType(vmCreateModel.getType());
        vmSendCreateModel.setStepOrder(vmCreateModel.getStepOrder());
        vmSendCreateModel.setDetailType(vmCreateModel.getDetailType());
        vmSendCreateModel.setArch(vmCreateModel.getArch());
        vmSendCreateModel.setCloudScope(vmCreateModel.getCloudScope());
        vmSendCreateModel.setOrder(vmCreateModel.getOrder());
    }


    //直接发送已经完成的消息
    @Override
    public void handleEmptyVmModels(VmCreateModel vmCreateModel) throws Exception {

        LOGGER.warn("handleEmptyVmModels svcId={}",vmCreateModel.getSvcId());

        int maxStep = StageDetailTypeEnum.getMaxStepOrder();
        for (int i=1;i < maxStep;i++){

            StageDetailTypeEnum stageDetailTypeEnum = StageDetailTypeEnum.getStageDetailTypeEnumByType(i);
            vmCreateModel.setType(stageDetailTypeEnum.getParentType());
            vmCreateModel.setDetailType(stageDetailTypeEnum.getDetailType());

            updateStageMsg(vmCreateModel);
        }

//        if ((vmCreateModel.getCreateScalingDTO() != null)
//                &&(StringUtils.isNotEmpty(vmCreateModel.getCreateScalingDTO().getClusterNames()))) {
//            //安装完成之后，调用策略端口，通知安装完成
//            ScalingService scalingService = SpringUtil.getBean(ScalingService.class);
//            //传入node数量
//            //v1.5,传入策略
//            scalingService.createDefaultScalingGroupForNewCluster(vmCreateModel.getUserName(),
//                    vmCreateModel.getCreateScalingDTO().getClusterNames(),
//                    CloudVendorsEnum.getDescribeByCloudeType(vmCreateModel.getCreateScalingDTO().getProvider()),
//                    k8sClusterService.getK8sClusterNodeNum(vmCreateModel.getCreateScalingDTO().getProvider(),
//                                    vmCreateModel.getCreateScalingDTO().getClusterNames()),
//                    vmCreateModel.getCreateScalingDTO());
//        }

        vmCreateInfoMapper.updateWaitCreateInfo0(vmCreateModel.getSvcId());

        //删除记录
        endVmCreateInfo(vmCreateModel);
    }

    /**
     *
     * @param url
     * @param param
     * @return
     */
    private String doHttpPostReq(String url, StringEntity param){

        try {

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpPost httpPost = new HttpPost(url);
            httpPost.addHeader("Content-Type", "application/json;charset=UTF-8");
            httpPost.setHeader("Accept", "application/json");

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(5 * 60 * 1000).setConnectionRequestTimeout(5 * 60 * 1000)
                    .setSocketTimeout(5 * 60 * 1000).build();
            httpPost.setConfig(requestConfig);
            httpPost.setEntity(param);

            CloseableHttpResponse response = httpclient.execute(httpPost);
            return EntityUtils.toString(response.getEntity());

        }catch (Throwable e){
            LOGGER.error("vmRes doHttpPostReq fail url={}",url,e);
        }

        return null;
    }

    /**
     * 通知集群修改完成
     * @param notifyClusterInfoDTO
     */
    public void notifyClusterMigrate(NotifyClusterInfoDTO notifyClusterInfoDTO){

        String url =  SCALE_URL + "/sys/intelligent/update/cluster";

        try {

            LOGGER.info("#########################");
            LOGGER.info("notifyClusterMigrate svcId={} clusterName={}",
                    notifyClusterInfoDTO.getSvcId(),notifyClusterInfoDTO.getClusterName());

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpPost httpPost = new HttpPost(url);

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(5 * 60 * 1000).setConnectionRequestTimeout(5 * 60 * 1000)
                    .setSocketTimeout(5 * 60 * 1000).build();
            Header header = new BasicHeader(HttpHeaders.CONTENT_TYPE,"application/json");

            httpPost.setHeader(header);
            httpPost.setConfig(requestConfig);
            httpPost.setEntity(new StringEntity(JSON.toJSONString(notifyClusterInfoDTO)));


            try {
                CloseableHttpResponse response = httpclient.execute(httpPost);
            } catch (Throwable e) {

                LOGGER.error("notifyClusterMigrate execute fail svcId={} clustername={}",
                        notifyClusterInfoDTO.getSvcId(),notifyClusterInfoDTO.getClusterName(), e);
            }

        } catch (Throwable e) {

            LOGGER.error("notifyClusterMigrate execute fail svcId={} clustername={}",
                    notifyClusterInfoDTO.getSvcId(),notifyClusterInfoDTO.getClusterName(), e);
        }
    }

    /**
     * 发送状态信息
     * @param sendVmCreateModel
     */
    @Override
    public void updateStageMsg(VmCreateModel sendVmCreateModel){
        //扩缩容的svc不用通知
        if (StringUtils.contains(sendVmCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_SVC_PRE)
                ||StringUtils.contains(sendVmCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)){
            return;
        }

        String url =  SCALE_URL + UPDATE_STAGE_PATH;

        try {

            //如果errorMsg过长，进行截断
            if(StringUtils.isNotEmpty(sendVmCreateModel.getErrorMsg())
                &&(sendVmCreateModel.getErrorMsg().length() > 300)){
                sendVmCreateModel.setErrorMsg(sendVmCreateModel.getErrorMsg().substring(0,300));
            }

            LOGGER.info("#########################");
            LOGGER.info("update stage svcId={} step={}",sendVmCreateModel.getSvcId(),sendVmCreateModel.getDetailType());

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpPost httpPost = new HttpPost(url);

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(5 * 60 * 1000).setConnectionRequestTimeout(5 * 60 * 1000)
                    .setSocketTimeout(5 * 60 * 1000).build();
            Header header = new BasicHeader(HttpHeaders.CONTENT_TYPE,"application/json");

            httpPost.setHeader(header);
            httpPost.setConfig(requestConfig);
            httpPost.setEntity(new StringEntity(JSON.toJSONString(sendVmCreateModel)));


            //重试三次
            for (int i = 0; i < 3; i++) {
                try {
                    CloseableHttpResponse response = httpclient.execute(httpPost);
                    if (response.getStatusLine().getStatusCode() == 200) {
                        break;
                    }
                } catch (Throwable e) {

                    LOGGER.error("updateStageMsg execute fail svcId={} type={} stepOrder={}",
                            sendVmCreateModel.getSvcId(), sendVmCreateModel.getType(),sendVmCreateModel.getStepOrder(), e);
                }
            }

        } catch (Throwable e) {

            LOGGER.error("updateStageMsg fail svcId={} type={} stepOrder={}", sendVmCreateModel.getSvcId(), sendVmCreateModel.getType()
                    ,sendVmCreateModel.getStepOrder(), e);
        }
    }

    /**
     * 发送状态信息
     * @param vmCreateModel
     */
    private void updateStageMsg(VmCreateModel vmCreateModel,String k8sClusterName,String cloudVendor){

        //扩缩容的svc不用通知
        if (StringUtils.contains(vmCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_SVC_PRE)){
            return;
        }

        String url =  SCALE_URL + UPDATE_STAGE_PATH;
        VmCreateModel sendVmCreateModel = new VmCreateModel();

        if (StringUtils.equals(vmCreateModel.getType(),CreateVmStepEnum.K8s.getDescribe())){
            //处理发送的信息
            if ((vmCreateModel.getVmModels() != null)
                    &&(!vmCreateModel.getVmModels().isEmpty())){

                List<Map<String,String>> extranInfo = new ArrayList<>();
                for (VmModel vmModel:vmCreateModel.getVmModels()){
                    Map<String,String> param = new HashMap<>();
                    param.put("name",k8sClusterName);
                    param.put("providerName",cloudVendor);
                    param.put("ip",vmModel.getExternalIp());
                    param.put("innerIp",vmModel.getInnerIp());
                    extranInfo.add(param);
                }
                sendVmCreateModel.setExtraInfo(extranInfo);
            }
        }

        try {

            CloseableHttpClient httpclient = HttpClients.createDefault();

            HttpPost httpPost = new HttpPost(url);

            RequestConfig requestConfig = RequestConfig.custom()
                    .setConnectTimeout(5 * 60 * 1000).setConnectionRequestTimeout(5 * 60 * 1000)
                    .setSocketTimeout(5 * 60 * 1000).build();
            Header header = new BasicHeader(HttpHeaders.CONTENT_TYPE,"application/json");

            sendVmCreateModel.setSvcId(vmCreateModel.getSvcId());
            if ((vmCreateModel.getErrorMsg() != null)
                    &&(vmCreateModel.getErrorMsg().length() > 100)) {
                sendVmCreateModel.setErrorMsg(vmCreateModel.getErrorMsg().substring(0,100));
            }else {
                sendVmCreateModel.setErrorMsg(vmCreateModel.getErrorMsg());
            }
            sendVmCreateModel.setType(vmCreateModel.getType());

            httpPost.setHeader(header);
            httpPost.setConfig(requestConfig);
            httpPost.setEntity(new StringEntity(JSON.toJSONString(sendVmCreateModel)));


            //重试三次
            for (int i = 0; i < 3; i++) {
                try {
                    CloseableHttpResponse response = httpclient.execute(httpPost);
                    if (response.getStatusLine().getStatusCode() == 200) {
                        break;
                    }
                } catch (Throwable e) {

                    LOGGER.error("updateStageMsg execute fail svcId={} type={}",
                            vmCreateModel.getSvcId(), vmCreateModel.getType(), e);
                }
            }

        } catch (Throwable e) {

            LOGGER.error("updateStageMsg fail svcId={} type={}", vmCreateModel.getSvcId(), vmCreateModel.getType(), e);
        }
    }

    /**
     *
     * @param cloudVendors
     * @param k8sClusterName
     * @param nodeName
     * @param osImage
     * @param cpuNum
     * @param memNum
     * @param appType
     * @throws Exception
     */
    @Override
    public void addNewResource(String cloudVendors, String k8sClusterName, String nodeName, String osImage, int cpuNum, int memNum, String appType) throws Exception {
    }

    /**
     *
     * @param cloudVendors
     * @param k8sClusterName
     * @param nodeNum
     * @param osImage
     * @param cpuNum
     * @param memNum
     * @param appType
     * @throws Exception
     */
    @Override
    public void addNewResource(String cloudVendors, String k8sClusterName, int nodeNum, String osImage, int cpuNum, int memNum, String appType) throws Exception {
    }

    /**
     * 删除虚拟机
     * @param vmModels
     */
    @Override
    public void delVms(List<VmModel> vmModels) {

        //根据vmId构建删除的结构
        DelVmSendModel delVmSendModel = new DelVmSendModel();
        delVmSendModel.setVms(new ArrayList<>());
        for (VmModel vmModel:vmModels) {

            DelVmInfoModel delVmInfoModel = new DelVmInfoModel();
            delVmSendModel.setUserId(vmModel.getUserId());
            delVmSendModel.setBusinessId(vmModel.getBusinessId());
            delVmSendModel.setAccountName(vmModel.getAccountName());
            delVmInfoModel.setVmId(vmModel.getVmId());
            delVmInfoModel.setCloudType(vmModel.getCloudType());
            delVmSendModel.getVms().add(delVmInfoModel);
        }

        String url = VM_CONTROLER_URL + "/vm/v1/delete";
        String param = JSON.toJSONString(delVmSendModel);
        if (!delVmSendModel.getVms().isEmpty()){

            try {
                LOGGER.warn("delVms param={}",param);
                String delResult = doHttpPostReq(url, new StringEntity(JSON.toJSONString(delVmSendModel)));
                LOGGER.warn("delVms respone result={}",delResult);
            }catch (Throwable e){
                LOGGER.error("delVms del vm fail uid={} bizId={}",
                        delVmSendModel.getUserId(),delVmSendModel.getBusinessId(),e);
            }
        }

    }

    /**
     *
     * @param cloudVendors
     * @param k8sClusterName
     * @param appType
     * @param delPolicy
     * @param delNodeNum
     * @return
     */
    @Override
    public boolean delResource(String cloudVendors, String k8sClusterName, String appType, String delPolicy, int delNodeNum) {

        LOGGER.warn("delResource k8sname={} apptype={} policy={} num={}",k8sClusterName,appType,delPolicy,delNodeNum);
        K8sClusterBasicModel k8sClusterBasicModel =
                multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);

        //删除最后进入的几个vm
        List<VmModel> delVmModels = new ArrayList<>();

        List<VmModel> vmModels = k8sClusterBasicModel.getVmModels();
        //删除策略有LIFO，FIFO
        if (StringUtils.equals(delPolicy,"FIFO")){
            vmModels.sort((vmModel, t1) -> (int) (vmModel.getCreateTimeStamp() - t1.getCreateTimeStamp()));

        }else {
            
            vmModels.sort((vmModel, t1) -> (int) (t1.getCreateTimeStamp() - vmModel.getCreateTimeStamp()));
        }
        Map<String,String> delVmMap = new HashMap<>();

        //找到所有要删除vm
        for (VmModel vmModel:vmModels){
            if (StringUtils.equals(vmModel.getK8sCluterRole(),CnapCloudMgmtConstants.CNAP_VM_WORKER)
                    &&StringUtils.equalsIgnoreCase(vmModel.getSvcIdLabelValue(),appType)){
                delVmModels.add(vmModel);
            }

            if (delVmModels.size() == delNodeNum){
                break;
            }
        }

        LOGGER.warn("delResource del num" +
                " k8sname={} apptype={} num={}",k8sClusterName,appType,delVmModels.size());

        VmModel masterModel = new VmModel();
        for (VmModel vmModel:vmModels){
            if (StringUtils.equals(vmModel.getK8sCluterRole(),CnapCloudMgmtConstants.CNAP_VM_MASTER)){
                masterModel = vmModel;
                break;
            }
        }

        for (VmModel vmModel:delVmModels){
            LOGGER.warn("delResource delVm" +
                    " k8sname={} apptype={} vmIp={}",k8sClusterName,appType,vmModel.getInnerIp());

            try {

                k8sClusterService.delK8sClusterNode(k8sClusterBasicModel,
                        CnapCloudMgmtConstants.CNAP_NODE_PREFIX + vmModel.getInnerIp());

                //删除文件中的配置
                cnapSshService.delHostMumo(masterModel,vmModel);
                //删除免密登陆配置
                cnapSshService.delInnerIpNoPasswdSsh(masterModel,vmModel);

                delVmMap.put(vmModel.getVmId(),vmModel.getVmId());

            }catch (Exception e){
                LOGGER.error("delResource faile k8sname={} nodeip={}",k8sClusterBasicModel.getK8sClusterName(),
                        vmModel.getInnerIp(),e);
                return false;
            }
        }

        //更新vmjson
        List<VmModel> newVmModels = new ArrayList<>();
        for (VmModel vmModel:k8sClusterBasicModel.getVmModels()){
            if (delVmMap.containsKey(vmModel.getVmId())){
                continue;
            }
            newVmModels.add(vmModel);
        }
        String vmJson = JSONObject.toJSONString(newVmModels);
        multiCloudMgmtService.updateVmJson(k8sClusterBasicModel,vmJson);

        //删除节点
        delVms(delVmModels);

        return true;
    }

    /**
     * 规格匹配接口
     * url： /select_flavor
     * 请求格式
     * {
     *     "require_resources":{"cpu_usage":2, "memory_usage":4},
     *     "region_id": "cn-shanghai",
     *     "zone_id": "cn-shanghai-b",
     *     "cloud_type": "Aliyun"
     * }
     * 响应
     * {
     *     "create_vm_info": [
     *         {
     *             //注意，缺少name，需要补充
     *             "bandwidth": 10,
     *             "chargeCount": 6,
     *             "chargeUnit": "Month",
     *             "cloudType": "Aliyun",
     *             "dataDisks": [
     *                 {
     *                     "dataDiskSize": 50,
     *                     "dataDiskType": "cloud_efficiency"
     *                 }
     *             ],
     *             "eipChargeType": "PayByTraffic",
     *             "flavorId": "ecs.t5-c1m2.large",
     *             "imageId": "ubuntu_18_04_x64_20G_alibase_20221104.vhd",
     *             "regionId": "cn-beijing",
     *             "sysDiskSize": 100,
     *             "vmChargeType": "PostPaid",
     *             "zoneId": "cn-beijing-f"
     *         }
     *     ]
     *   //list格式，如果需要多个规格匹配，则返回多个规格
     * }
     * @param cloudVendors
     * @param k8sClusterName
     * @param cpuUsage
     * @param memUsage
     * @param appType
     * @return
     */
    @Override
    public int addNewResourceForScale(String cloudVendors, String k8sClusterName, int cpuUsage, int memUsage,String appType) {

        //这里的appType是svcId
        LOGGER.warn("addNewResourceForScale name={} cpu={} mem={} appType={}",
                k8sClusterName,cpuUsage, memUsage,appType);
        K8sClusterBasicModel k8sClusterBasicModel =
                multiCloudMgmtService.getK8sClusterInfoByName(k8sClusterName);

        //看看svc是不是暂停状态，是暂停状态不执行扩缩容，暂停状态就是svc不存在
        if(k8sClusterService.checkSvcSuspend(k8sClusterBasicModel,appType)){
            LOGGER.warn("addNewResourceForScale svc is suspend name={} cpu={} mem={} appType={}",
                    k8sClusterName,cpuUsage, memUsage,appType);
            return -1;
        }



        //先去算法那拿一下相关的信息
        int userId = k8sClusterBasicModel.getVmModels().get(0).getUserId();
        ResourceSelectFlavorModel resourceSelectFlavorModel = new ResourceSelectFlavorModel();
        resourceSelectFlavorModel.setCloud_type(
                CloudVendorsEnum.getCloudeTypeByDescribe(k8sClusterBasicModel.getCloudVendor()));
        resourceSelectFlavorModel.setZone_id(k8sClusterBasicModel.getZoneId());
        resourceSelectFlavorModel.setRegion_id(k8sClusterBasicModel.getRegionId());
        resourceSelectFlavorModel.setRequire_resources(new HashMap<>());
        resourceSelectFlavorModel.getRequire_resources().put("cpu_usage",cpuUsage);
        resourceSelectFlavorModel.getRequire_resources().put("memory_usage",memUsage);
        resourceSelectFlavorModel.setCommerce_type(CloudVendorsEnum.getCloudScopeByCloudeType(
                resourceSelectFlavorModel.getCloud_type()));
        resourceSelectFlavorModel.setAccount_name(k8sClusterBasicModel.getAccountName());

        String param = JSON.toJSONString(resourceSelectFlavorModel);

        LOGGER.warn("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
        LOGGER.warn("addNewResourceForScale select_flavor param={}",param);

        List<VmModel> vmModels = new ArrayList<>();
        String url = ALG_URL +  "/select_flavor";
        try {
            String result = doHttpPostReq(url, new StringEntity(param));

            /**
             * ut测试代码
             * 构造一个返回的数据
             * 用于校验
             * 判断依据是k8sClusterName
             */
            if (StringUtils.equals(k8sClusterBasicModel.getK8sClusterName(),CnapCloudMgmtConstants.TEST_ADMIN)){
                FlavorResourceModel testFlavorResourceModel = new FlavorResourceModel();
                testFlavorResourceModel.setCreate_vm_info(new ArrayList<>());
                testFlavorResourceModel.getCreate_vm_info().add(new VmModel());
                result = JSONObject.toJSONString(testFlavorResourceModel);
            }
            //ut end

            if (StringUtils.isNotEmpty(result)) {

                FlavorResourceModel flavorResourceModel
                        = JSONObject.parseObject(result, FlavorResourceModel.class);
                vmModels = flavorResourceModel.getCreate_vm_info();

            } else {

                throw new Exception("no result for Scale vm clouder=" + cloudVendors + " k8sName=" + k8sClusterName);
            }
            LOGGER.warn(" Scale vm return vmResult={}",JSONObject.toJSONString(vmModels));
        }catch (Throwable e){

            LOGGER.error("addNewResourceForScale req alg fail name={}",k8sClusterName,e);
            return -1;
        }

        VmCreateModel vmCreateModel = new VmCreateModel();
        vmCreateModel.setVmModels(vmModels);
        String svcId = CnapCloudMgmtConstants.SCALE_SVC_PRE + appType;
        //获取一下，看看这个svcId是不是已经存在
        VmCreateModel vmCreateModelOld = vmCreateInfoMapper.getCreateInfoBySvcId(svcId);
        if (vmCreateModelOld != null){

            //如果errormsg不为空，就说明要删掉，如果errormsg为空，那就是还在扩容，要等待这个弄完
            if(StringUtils.isEmpty(vmCreateModelOld.getErrorMsg())){

                LOGGER.warn("addNewResourceForScale svc is in scale svcId={} type={} createTime={}",
                        vmCreateModelOld.getSvcId(),vmCreateModelOld.getType(),vmCreateModelOld.getCreateTime());
                return -1;
            }else {

                LOGGER.warn("addNewResourceForScale svc is not empty svcId={} type={} createTime={} errorMsg={}",
                        vmCreateModelOld.getSvcId(),vmCreateModelOld.getType(),vmCreateModelOld.getCreateTime(),
                        vmCreateModelOld.getErrorMsg());
                vmCreateInfoMapper.delRecord(svcId);
            }
        }

        vmCreateModel.setSvcId(svcId);

        if (StringUtils.containsIgnoreCase(k8sClusterBasicModel.getZoneId(),CnapCloudMgmtConstants.FORCED_ISOLATION)){

            vmCreateModel.setUserName(k8sClusterBasicModel.getCreateUser()+CnapCloudMgmtConstants.FORCED_ISOLATION);

        }else{
            vmCreateModel.setUserName(k8sClusterBasicModel.getCreateUser());
        }

        //获取svcId对应的名字
        MicroServicePO microServicePO =
                microSvcMapper.getMicroServiceByUserNameId(k8sClusterBasicModel.getCreateUser(),appType);
        if (microServicePO == null) {
            LOGGER.error("addNewResourceForScale get svc info fail svcId={} userName={} ",
                    appType,k8sClusterBasicModel.getCreateUser());
        }

        //设置accountName
        String accountName = k8sClusterBasicModel.getVmModels().get(0).getAccountName();
        vmCreateModel.setAccountName(accountName);

        for (VmModel vmModel:vmCreateModel.getVmModels()){
            vmModel.setBusinessId(k8sClusterBasicModel.getBusinessId());
            //先写死ai
            vmModel.setCategory("ai");
            vmModel.setUserId(userId);
            //v2.0 appType就是svcId
            vmModel.setSvcIdLabelValue(appType);
            vmModel.setSvcNameLabelValue(microServicePO.getServiceName());
            vmModel.setArch(CnapCloudMgmtConstants.ARCH_CPU);
            vmModel.setAccountName(accountName);

        }
        vmCreateModel.setType(StageTypeEnum.INIT.getType());
        vmCreateModel.setCreateTime(new Date());
        vmCreateModel.setUserId(userId);
        vmCreateModel.setForcedIsolation(false);
        vmCreateModel.setWaitSvcId("0");
        vmCreateModel.setAppId((new Date()).toString());
        //现在只有cpu配置了扩缩容
        if (StringUtils.isEmpty(vmCreateModel.getArch())){
            vmCreateModel.setArch(CnapCloudMgmtConstants.ARCH_CPU);
        }
        vmCreateInfoMapper.saveVmCreateRecord(vmCreateModel);

        return vmCreateModel.getVmModels().size();
        
    }

    /**
     * 步骤一删除，删除信息，删除节点
     * 步骤二删除，删除信息，删除节点。
     * 步骤三删除，如果集群不存在，那就删除信息，删除节点
     * 如果集群存在，那就结束步骤，不对节点做操作
     * 步骤四删除，不做动作。
     *
     */
    @Override
    public void delSvcToVmCreate(String svcId, String type) throws Exception {

        LOGGER.warn("delSvcToVmCreate to del svcId={} type={}",svcId,type);

        VmCreateModel vmCreateModel = null;
        //如果已经结束的svc，在数据库中已经加了其他标识了
        if (StringUtils.equals(type,StageTypeEnum.DISTRIBUTE.getType())
                ||StringUtils.equals(type,StageTypeEnum.END.getType())) {

            //找到最新的不是扩容的svc的信息
            List<VmCreateModel> vmCreateModels = vmCreateInfoMapper.getCreateInfoLikeSvcId(svcId);
            if ((vmCreateModels != null)&&(!vmCreateModels.isEmpty())){

                for (VmCreateModel tmpCreateModel:vmCreateModels){

                    //如果是扩容的就不管他
                    if (StringUtils.contains(tmpCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_SVC_PRE)){

                        continue;
                    }
                    if (StringUtils.contains(tmpCreateModel.getSvcId(),CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)){

                        continue;
                    }
                    if (StringUtils.contains(tmpCreateModel.getSvcId(),CnapCloudMgmtConstants.MIGRATE_VM)){

                        continue;
                    }

                    //找到最新的一个新建
                    if (vmCreateModel == null){
                        vmCreateModel = tmpCreateModel;
                    }else {

                        if (tmpCreateModel.getCreateTime().getTime()
                                > vmCreateModel.getCreateTime().getTime()){
                            vmCreateModel = tmpCreateModel;
                        }
                    }
                }
            }

        }else{

            vmCreateModel = vmCreateInfoMapper.getCreateInfoBySvcId(svcId);

        }
        /**
         * 为null可能情况
         * svcId错了
         * 步骤已经结束，svcId在数据库里面的存储已经加了时间
         */
        if (vmCreateModel == null){
            LOGGER.warn("delSvcToVmCreate to del is empty svcId={} type={}",svcId,type);
            return;
        }

        LOGGER.warn("delSvcToVmCreate to del begin svcId={}",vmCreateModel.getSvcId());

        //没有passwd代表没有开通成功，要直接删除信息就好
        List<VmModel> vmModels = vmCreateModel.getVmModels();
        if (StringUtils.isEmpty(vmModels.get(0).getPasswd())){

            LOGGER.warn("delSvcToVmCreate to del no passwd svcId={}",vmCreateModel.getSvcId());

            vmCreateInfoMapper.delRecord(svcId);
            return;
        }

        //有passwd，但是不是集群的master节点的话，就直接删除，所有节点，然后删掉该条svc
        List<K8sClusterModel> k8sClusterModels = multiCloudMgmtService.getK8sClusterInfosByZone(
                vmCreateModel.getVmModels().get(0),
                vmCreateModel.getUserName());

        //如果集群不为空，所有非master节点都要被删除
        if ((k8sClusterModels != null)&&(!k8sClusterModels.isEmpty())){
            for (K8sClusterModel k8sClusterModel:k8sClusterModels) {

                for (VmModel vmModel:vmModels) {

                    LOGGER.warn("delSvcToVmCreate to del vmModel svcId={} clusterName={}" +
                            " ip={}",vmCreateModel.getSvcId(),k8sClusterModel.getK8sClusterName(),
                            vmModel.getInnerIp());

                    k8sClusterService.delK8sClusterNode(k8sClusterModel.getK8sClusterIp(),
                            CnapCloudMgmtConstants.CNAP_NODE_PREFIX + vmModel.getInnerIp());
                }
            }
        }else{

            LOGGER.warn("delSvcToVmCreate to del k8sCluster is empty svcId={}",vmCreateModel.getSvcId());

        }

        //删除svc
        vmCreateInfoMapper.delRecord(vmCreateModel.getSvcId());

    }

    @Override
    public void removeCreateVmProcess(String svcId) {

        //把数据库中的svcId相关都停掉(所有不是end状态的)
        List<VmCreateModel> vmCreateModels = vmCreateInfoMapper.getCreateInfoLikeSvcIdNoEnd(svcId);

        if ((vmCreateModels != null)&&(!vmCreateModels.isEmpty())){

            for (VmCreateModel vmCreateModel:vmCreateModels){
                endVmCreateInfo(vmCreateModel);
            }
        }

        cacheService.delVmCreate(svcId);
        cacheService.removeStopSvcId(svcId);
    }

    //处理返回svcId
    private String getRealSvcId(String svcId){

        if (StringUtils.contains(svcId,CnapCloudMgmtConstants.SCALE_SVC_PRE)){

            return svcId.split(CnapCloudMgmtConstants.SCALE_SVC_PRE)[1];
        }

        if (StringUtils.contains(svcId,CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)){

            return svcId.split(CnapCloudMgmtConstants.SCALE_CLUSTER_PRE)[1];
        }

        if (StringUtils.contains(svcId,CnapCloudMgmtConstants.MIGRATE_VM)){

            return svcId.split(CnapCloudMgmtConstants.MIGRATE_VM)[1];
        }

        return svcId;
    }
}
